import time
import requests
# 导入数据库模块
import pymysql
# 导入正则模块
import re
# 导入线程模块
import threading
# 导入数据库配置文件
import config
#
import random

# 设置数据库连接信息
host = config.HOST
post = config.PORT
username = config.USERNAME
password = config.PASSWORD
database = config.DATABASE

# 定义一个锁，以确保线程安全
tlock=threading.Lock()


db = pymysql.connect(
    host="localhost",
    port=3306,
    user=username,
    password=password,
    db=database,
    charset='utf8'
)

# 拿到游标
cursor = db.cursor()

# 薪资处理函数
def salary_alter(salary_str):
    if salary_str==0:
        return [0,0]
    # 移除"·XX薪"的后缀
    salary_str = re.sub(r'·\d+薪', '', salary_str)
    # 将薪资范围处理成阿拉伯数字
    def replace_salary(match):
        # 提取匹配的内容
        min_salary = int(match.group(1)) * 1000  # 低薪资数
        max_salary = int(match.group(2)) * 1000  # 高薪资数
        return f'{min_salary}-{max_salary}'  # 返回列表

    # 使用正则表达式处理输入字符串，替换薪资范围
    result = re.sub(r'(\d+)-(\d+)k', replace_salary, salary_str)
    new_result=result.split('-')
    return [new_result[0],new_result[1]]

# 时间处理函数
def format_time(input_time):
    # 检查输入长度
    if len(input_time) != 14:
        return input_time
    # 提取年月日和时分秒
    year = input_time[0:4]  # 前四位为年份
    month = input_time[4:6]  # 第五和第六位为月份
    day = input_time[6:8]    # 第七和第八位为日期
    hour = input_time[8:10]   # 第九和第十位为小时
    minute = input_time[10:12] # 第十一和第十二位为分钟
    second = input_time[12:14] # 第十三和第十四位为秒

    formatted_time = f"{year}-{month}-{day} {hour}:{minute}:{second}"
    return formatted_time

""""获取加密参数的方法"""
# 获取ckId值
def getckid(length=8):
    # 获取当前的时间戳（毫秒）
    t = int(time.time() * 1000)
    # 确保长度不小于 8
    n = max(length, 8)
    # 生成初始的字符串模板
    r = 'x' * n
    # 替换与生成的随机数
    i = ''
    for char in r:
        n = (t + 36 * random.random()) % 36
        t = t // 36
        # 这里确保 n 的值有效并转换为字符串
        if char == 'x':
            i += str(n)  # 确保 n 是数字
        else:
            i += str((n & 3) | 8)
            # 过滤掉非数字字符
    valid_chars = [c for c in i if c.isdigit()]
    # 转换为36进制字符串
    return ''.join([to_base36(int(c)) for c in valid_chars])
def to_base36(number):
    if number < 10:
        return str(number)
    else:
        return chr(number - 10 + ord('a'))
""""获取加密参数的方法"""

def write_db(data):
    """写入数据库"""
    #print(data)
    try:
        tlock.acquire()
        # rows变量得到数据库中被影响的数据行数。
        rescoun = cursor.execute(
            "insert into data (titles, company,compScale,city,area, min_salary, max_salary, date, degree, exper, content_link) values(%s,%s, %s,%s,%s,%s,%s, %s,%s, %s,%s)", data)
        # 向数据库提交
        db.commit()
        tlock.release()
        # 如果没有commit()，库中字段已经向下移位但内容没有写进，可是自动生成的ID会自动增加。
        #print('插入数据成功')

    except:
        # 发生错误时回滚
        db.rollback()
        tlock.release()
        print('插入失败')

# 获取城市
def get_cityid(city):
    try:
        tlock.acquire()
        cursor.execute("select city_id from city where city_name='%s'" % city)
        city_id = cursor.fetchone()
        # 向数据库提交
        db.commit()
        tlock.release()
        if city_id == None:
            return '410' #输入地区错误则默认全国地区
        else:
            return city_id[0]
    except:
        # 发生错误时回滚
        db.rollback()
        print("查询失败")
        return 0


# 获取数据
def get_data(keyword,city_id,startPage,endPage):
    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Connection': 'keep-alive',
        'Content-Type': 'application/json;charset=UTF-8',
        'Cookie': '__gc_id=21bf86e2c7764b74a3509f70236bcf96; _ga=GA1.1.926860482.1732004483; __uuid=1732004482884.32; XSRF-TOKEN=ydVMWL5zRSakGd7bwlX3OQ; acw_tc=1a0c638417344564463472748e0037f7ee437870b267fc0db4f77ee5833671; __tlog=1734456446252.11%7C00000000%7C00000000%7Cs_o_007%7Cs_o_007; Hm_lvt_a2647413544f5a04f00da7eee0d5e200=1732004483,1732199436,1734456446; HMACCOUNT=9D4F3C34992F27D8; _ga_54YTJKWN86=GS1.1.1734456445.5.1.1734456454.0.0.0; Hm_lpvt_a2647413544f5a04f00da7eee0d5e200=1734456455; __session_seq=5; __tlg_event_seq=36',
        'Origin': 'https://www.liepin.com',
        'Referer': 'https://www.liepin.com/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
        'X-Client-Type': 'web',
        'X-Fscp-Bi-Stat': '{"location": "https://www.liepin.com/zhaopin/?inputFrom=www_index&workYearCode=0&key=%E4%BC%9A%E8%AE%A1&scene=hot_search&ckId=5l3hnxwox4w7ajq942yx70dt1t31xbra&dq="}',
        'X-Fscp-Fe-Version': '',
        'X-Fscp-Std-Info': '{"client_id": "40108"}',
        'X-Fscp-Trace-Id': '9f33bcc3-5747-4393-8462-dca841a21334',
        'X-Fscp-Version': '1.1',
        'X-Requested-With': 'XMLHttpRequest',
        'X-XSRF-TOKEN': 'ydVMWL5zRSakGd7bwlX3OQ',
        'sec-ch-ua': '"Microsoft Edge";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
    }
    ckId=getckid(32)#获取加密的参数
    # 批量采集
    for page in range(startPage, endPage+1):
        print(f'正在采集第{page}页数据')
        data = {"data": {
            "mainSearchPcConditionForm": {"city":city_id, "dq":city_id, "pubTime": "3", "currentPage": page,
                                          "pageSize": 40, "key":keyword, "suggestTag": "", "workYearCode": "0",
                                          "compId": "", "compName": "", "compTag": "", "industry": "", "salary": "",
                                          "jobKind": "", "compScale": "", "compKind": "", "compStage": "",
                                          "eduLevel": ""},
            "passThroughForm": {
                "sfrom": "search_job_pc",
                "scene": "page",
                "ckId": ckId,
                "skId": "0xh2nrkjwdlh0uxb25g4nri253r3q4a9",
                "fkId": "zqwzv5jrlu63uc51w79vpnoq4gqyke4b"
            }
        }
        }
        url = "https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job"
        response = requests.post(url, headers=headers, json=data)
        json_data = response.json()
        try:
            jobCardList = json_data['data']['data']['jobCardList']
        except:
            print(f"第{page}页未发现数据")
            # exit()
        # if not jobCardList:
        #     print(f"第{page}页未发现数据")
        #     # exit()
        # 遍历列表提取数据、
        for jobcard in jobCardList:
            address = jobcard['job']['dq'].split('-')
            if len(address) == 2:
                # 城市
                city = address[0]
                # 区域
                area = address[1]
            else:
                city = address[0]
                area = '未知'
            salary = jobcard['job']['salary']
            if '面议' in salary:
                salary = 0
            # 处理薪资表示
            reulst_salary=[]
            reulst_salary=salary_alter(salary)
            # 最小薪资
            min_salary=reulst_salary[0]
            # 最大薪资
            max_salary = reulst_salary[1]
            # 职位
            titles=jobcard['job']['title']
            if max_salary==0:
                titles=titles+'【薪资面议】'
            # 公司
            company=jobcard['comp']['compName']
            # 公司人数
            if 'compScale' in jobcard['comp']:
                compScale=jobcard['comp']['compScale']
            else:
                compScale='未知'
            if compScale=='':
                compScale='未知'
            # 招聘详情页
            content_link=jobcard['job']['link']
            # 学历
            if 'requireEduLevel' in jobcard['job']:
                degree=jobcard['job']['requireEduLevel']
            else:
                degree='学历不限'
            if '统招' in degree:
                degree=re.sub(r'统招','',degree)
            # 经验
            if 'requireWorkYears' in jobcard['job']:
                exper=jobcard['job']['requireWorkYears']
            else:
                exper="无工作经验"
            # 发布时间
            date= format_time(jobcard['job']['refreshTime'])
            # 数据列表
            item = [titles, company,compScale, city, area, min_salary, max_salary, date, degree, exper, content_link]
            # 写进数据库
            write_db(item)



def data_clr():
    # 清空data
    try:
        tlock.acquire()
        query = "truncate table `data`"
        cursor.execute(query)
        db.commit()
        tlock.release()
        print('data原表已清空')
    except Exception as aa:
        print(aa)
        print('无data表！')

def main(kw, city,startPage,endPage):
    """
    startPage:开始页
    endPage：结束页
    """
    print(kw)
    print(city)
    city_id = get_cityid(city)
    print(city_id)
    data_clr()
    start_time = time.time()
    get_data(kw,city_id,startPage,endPage)
    end_time = time.time()
    print(f'用时{end_time - start_time}')

if __name__ == '__main__':
    start_time=time.time()
    main('会计','广东',1,10)
    end_time=time.time()
    print(f'用时{end_time-start_time}')
